{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5a83e92e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('1.15.1', '2.2.3', '2.2.3')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from BradleyTerry import SubjectiveScores\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "\n",
    "scipy.__version__, np.__version__, pd.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9322ad6a-fd79-4d26-98ac-11b4aa284784",
   "metadata": {},
   "source": [
    "# First Subjective Phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cecaf98c-5ae4-415d-bd1e-8af4fb597e9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./subjective_votes/phase1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cf3e485a-0e97-46b9-be23-e7a2acb723ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Bradley-Terry\n",
      "Questions: 30600\n",
      "Sequences: 20\n",
      "Methods: 18\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>subjective_score</th>\n",
       "      <th>confidence_95_to_original</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>230090</td>\n",
       "      <td>3.5958425865478105</td>\n",
       "      <td>0.1002768947115975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>230089</td>\n",
       "      <td>3.45275199183467</td>\n",
       "      <td>0.09944393711204051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>230088</td>\n",
       "      <td>3.443616517976187</td>\n",
       "      <td>0.09939929722488561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>230087</td>\n",
       "      <td>3.42782593585722</td>\n",
       "      <td>0.09932451712997008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>230086</td>\n",
       "      <td>3.335548274972779</td>\n",
       "      <td>0.09894726752258745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>230085</td>\n",
       "      <td>3.2715483749371583</td>\n",
       "      <td>0.09874489182582874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>230104</td>\n",
       "      <td>3.2435436600561514</td>\n",
       "      <td>0.09867146160038566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>230103</td>\n",
       "      <td>3.1894287777219716</td>\n",
       "      <td>0.09855551874842546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>230082</td>\n",
       "      <td>3.1389748578496213</td>\n",
       "      <td>0.09847813663345167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>230102</td>\n",
       "      <td>3.088557671190826</td>\n",
       "      <td>0.09843037328802187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>230080</td>\n",
       "      <td>3.023840320154273</td>\n",
       "      <td>0.09841240707131635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>230079</td>\n",
       "      <td>2.9631248023727945</td>\n",
       "      <td>0.09844000263377607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>230078</td>\n",
       "      <td>2.9314889936428994</td>\n",
       "      <td>0.09847152075990712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>230077</td>\n",
       "      <td>2.928501016195832</td>\n",
       "      <td>0.09847510698800782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>230101</td>\n",
       "      <td>2.8877666134113094</td>\n",
       "      <td>0.09853452763253433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>230100</td>\n",
       "      <td>2.85890499335376</td>\n",
       "      <td>0.09858855448945933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Original</td>\n",
       "      <td>2.429182838417973</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>230074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.17786859296126228</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        name    subjective_score confidence_95_to_original\n",
       "0     230090  3.5958425865478105        0.1002768947115975\n",
       "1     230089    3.45275199183467       0.09944393711204051\n",
       "2     230088   3.443616517976187       0.09939929722488561\n",
       "3     230087    3.42782593585722       0.09932451712997008\n",
       "4     230086   3.335548274972779       0.09894726752258745\n",
       "5     230085  3.2715483749371583       0.09874489182582874\n",
       "6     230104  3.2435436600561514       0.09867146160038566\n",
       "7     230103  3.1894287777219716       0.09855551874842546\n",
       "8     230082  3.1389748578496213       0.09847813663345167\n",
       "9     230102   3.088557671190826       0.09843037328802187\n",
       "10    230080   3.023840320154273       0.09841240707131635\n",
       "11    230079  2.9631248023727945       0.09844000263377607\n",
       "12    230078  2.9314889936428994       0.09847152075990712\n",
       "13    230077   2.928501016195832       0.09847510698800782\n",
       "14    230101  2.8877666134113094       0.09853452763253433\n",
       "15    230100    2.85890499335376       0.09858855448945933\n",
       "16  Original   2.429182838417973                       0.0\n",
       "17    230074                 0.0       0.17786859296126228"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SubjectiveScores(df, save_path='./subjective_scores/phase1.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "769da5c4-6dba-4051-9720-73a5874ad8eb",
   "metadata": {},
   "source": [
    "# Second Subjective Phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e38b09cb-0249-4906-9520-8b03f44cfe01",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./subjective_votes/phase2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "289bc30a-e9d2-4a41-b25b-d01904f6ba6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Bradley-Terry\n",
      "Questions: 38000\n",
      "Sequences: 20\n",
      "Methods: 20\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>subjective_score</th>\n",
       "      <th>confidence_95_to_original</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>237831</td>\n",
       "      <td>6.791462467642389</td>\n",
       "      <td>0.09935187456603561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>237829</td>\n",
       "      <td>6.677294211784168</td>\n",
       "      <td>0.09875421855002087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>235858</td>\n",
       "      <td>6.668362412161126</td>\n",
       "      <td>0.09871341431010654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>235830</td>\n",
       "      <td>6.568951189377833</td>\n",
       "      <td>0.09831629697364314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>235721</td>\n",
       "      <td>6.550319763957455</td>\n",
       "      <td>0.09825339434646561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>235711</td>\n",
       "      <td>6.513149020383314</td>\n",
       "      <td>0.09813864856967128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>237825</td>\n",
       "      <td>6.471123175728678</td>\n",
       "      <td>0.09802606590190965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>235696</td>\n",
       "      <td>6.451510023113685</td>\n",
       "      <td>0.097979724159247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>237827</td>\n",
       "      <td>6.448248930678667</td>\n",
       "      <td>0.09797240041240775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>235735</td>\n",
       "      <td>6.424331484974785</td>\n",
       "      <td>0.09792200611280157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>235796</td>\n",
       "      <td>6.403156491964678</td>\n",
       "      <td>0.09788225911317146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>237826</td>\n",
       "      <td>6.399329527477752</td>\n",
       "      <td>0.09787556292720535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>237828</td>\n",
       "      <td>6.397144888992585</td>\n",
       "      <td>0.09787180723020086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>235712</td>\n",
       "      <td>6.298370979603544</td>\n",
       "      <td>0.09775269837007763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>237800</td>\n",
       "      <td>6.294023831985497</td>\n",
       "      <td>0.09774973456798192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>235691</td>\n",
       "      <td>6.076598986809072</td>\n",
       "      <td>0.09784830374118564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>237830</td>\n",
       "      <td>5.744326686717296</td>\n",
       "      <td>0.09897175394493729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>237727</td>\n",
       "      <td>5.598513151190291</td>\n",
       "      <td>0.0998680458218433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Original</td>\n",
       "      <td>5.316620255702701</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>237851</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.7240144055932517</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        name   subjective_score confidence_95_to_original\n",
       "0     237831  6.791462467642389       0.09935187456603561\n",
       "1     237829  6.677294211784168       0.09875421855002087\n",
       "2     235858  6.668362412161126       0.09871341431010654\n",
       "3     235830  6.568951189377833       0.09831629697364314\n",
       "4     235721  6.550319763957455       0.09825339434646561\n",
       "5     235711  6.513149020383314       0.09813864856967128\n",
       "6     237825  6.471123175728678       0.09802606590190965\n",
       "7     235696  6.451510023113685         0.097979724159247\n",
       "8     237827  6.448248930678667       0.09797240041240775\n",
       "9     235735  6.424331484974785       0.09792200611280157\n",
       "10    235796  6.403156491964678       0.09788225911317146\n",
       "11    237826  6.399329527477752       0.09787556292720535\n",
       "12    237828  6.397144888992585       0.09787180723020086\n",
       "13    235712  6.298370979603544       0.09775269837007763\n",
       "14    237800  6.294023831985497       0.09774973456798192\n",
       "15    235691  6.076598986809072       0.09784830374118564\n",
       "16    237830  5.744326686717296       0.09897175394493729\n",
       "17    237727  5.598513151190291        0.0998680458218433\n",
       "18  Original  5.316620255702701                       0.0\n",
       "19    237851                0.0        0.7240144055932517"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SubjectiveScores(df, save_path='./subjective_scores/phase2.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bdd3f9b-c2c5-4f3b-8e1c-97e7e8c9eec7",
   "metadata": {},
   "source": [
    "# Third Subjective Phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2598499b-65d5-4b36-a481-e52d382d6692",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./subjective_votes/phase3.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ecad72d6-cb8a-4eaa-a055-ccbc8b38f52c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Bradley-Terry\n",
      "Questions: 42000\n",
      "Sequences: 20\n",
      "Methods: 21\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>subjective_score</th>\n",
       "      <th>confidence_95_to_original</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>242892</td>\n",
       "      <td>2.2090909620704906</td>\n",
       "      <td>0.10364407192613018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>242922</td>\n",
       "      <td>2.024555848651111</td>\n",
       "      <td>0.10229708101687948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>242887</td>\n",
       "      <td>1.8128138229978719</td>\n",
       "      <td>0.10117032047207104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>242891</td>\n",
       "      <td>1.796926432125384</td>\n",
       "      <td>0.10110268877710012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>242894</td>\n",
       "      <td>1.7642854793775302</td>\n",
       "      <td>0.1009708913304353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>242899</td>\n",
       "      <td>1.642002802433551</td>\n",
       "      <td>0.10056110334044695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>242839</td>\n",
       "      <td>1.6199161452062516</td>\n",
       "      <td>0.1005009789055334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>242888</td>\n",
       "      <td>1.5668574215426232</td>\n",
       "      <td>0.10037363506698392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>242864</td>\n",
       "      <td>1.5463782018728207</td>\n",
       "      <td>0.10033089972020784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>242875</td>\n",
       "      <td>1.5359088251222925</td>\n",
       "      <td>0.10031042626093097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>242866</td>\n",
       "      <td>1.5074784895956812</td>\n",
       "      <td>0.10025950415467234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>242907</td>\n",
       "      <td>1.4551331204664717</td>\n",
       "      <td>0.10018355498811778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>242838</td>\n",
       "      <td>1.4381902146395513</td>\n",
       "      <td>0.10016390077317908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>242844</td>\n",
       "      <td>1.3328906716579765</td>\n",
       "      <td>0.10009564159858762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>242827</td>\n",
       "      <td>1.3043649559928427</td>\n",
       "      <td>0.10009313367895889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>242898</td>\n",
       "      <td>1.2566970324116542</td>\n",
       "      <td>0.10010419608851855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>242829</td>\n",
       "      <td>1.2425976147385693</td>\n",
       "      <td>0.10011113650219823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>242843</td>\n",
       "      <td>1.1814856533639653</td>\n",
       "      <td>0.10016068774495679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>242841</td>\n",
       "      <td>1.0943292501430686</td>\n",
       "      <td>0.10028665504182552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>242909</td>\n",
       "      <td>0.5610310431118666</td>\n",
       "      <td>0.10259348796602986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Original</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        name    subjective_score confidence_95_to_original\n",
       "0     242892  2.2090909620704906       0.10364407192613018\n",
       "1     242922   2.024555848651111       0.10229708101687948\n",
       "2     242887  1.8128138229978719       0.10117032047207104\n",
       "3     242891   1.796926432125384       0.10110268877710012\n",
       "4     242894  1.7642854793775302        0.1009708913304353\n",
       "5     242899   1.642002802433551       0.10056110334044695\n",
       "6     242839  1.6199161452062516        0.1005009789055334\n",
       "7     242888  1.5668574215426232       0.10037363506698392\n",
       "8     242864  1.5463782018728207       0.10033089972020784\n",
       "9     242875  1.5359088251222925       0.10031042626093097\n",
       "10    242866  1.5074784895956812       0.10025950415467234\n",
       "11    242907  1.4551331204664717       0.10018355498811778\n",
       "12    242838  1.4381902146395513       0.10016390077317908\n",
       "13    242844  1.3328906716579765       0.10009564159858762\n",
       "14    242827  1.3043649559928427       0.10009313367895889\n",
       "15    242898  1.2566970324116542       0.10010419608851855\n",
       "16    242829  1.2425976147385693       0.10011113650219823\n",
       "17    242843  1.1814856533639653       0.10016068774495679\n",
       "18    242841  1.0943292501430686       0.10028665504182552\n",
       "19    242909  0.5610310431118666       0.10259348796602986\n",
       "20  Original                 0.0                       0.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SubjectiveScores(df, save_path='./subjective_scores/phase3.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c80bfbd-83a1-4bd7-a006-de8813e40767",
   "metadata": {},
   "source": [
    "# PUBLIC TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cd985b4e-036e-40c4-8a95-c687d38e038a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./subjective_votes/final.csv')\n",
    "df = df[df['subset'] != 'private']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1dab8b82-8147-411d-81c5-589be5f18288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Bradley-Terry\n",
      "Questions: 33600\n",
      "Sequences: 120\n",
      "Methods: 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>subjective_score</th>\n",
       "      <th>confidence_95_to_original</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ShannonLab</td>\n",
       "      <td>1.7796967166398923</td>\n",
       "      <td>0.06620005999932352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DeepView</td>\n",
       "      <td>1.4816117259650852</td>\n",
       "      <td>0.06441957889511196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nobody</td>\n",
       "      <td>1.2730893875529137</td>\n",
       "      <td>0.06354890292970922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ChouPiJiang</td>\n",
       "      <td>1.0873058334025123</td>\n",
       "      <td>0.06301290394835951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>VQE</td>\n",
       "      <td>1.0429481551149549</td>\n",
       "      <td>0.0629176369270417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ByteMM</td>\n",
       "      <td>0.7156069244490271</td>\n",
       "      <td>0.06261091477571115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>TACO_SR</td>\n",
       "      <td>0.6184829741429617</td>\n",
       "      <td>0.06266025107009182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Original</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name    subjective_score confidence_95_to_original\n",
       "0   ShannonLab  1.7796967166398923       0.06620005999932352\n",
       "1     DeepView  1.4816117259650852       0.06441957889511196\n",
       "2       Nobody  1.2730893875529137       0.06354890292970922\n",
       "3  ChouPiJiang  1.0873058334025123       0.06301290394835951\n",
       "4          VQE  1.0429481551149549        0.0629176369270417\n",
       "5       ByteMM  0.7156069244490271       0.06261091477571115\n",
       "6      TACO_SR  0.6184829741429617       0.06266025107009182\n",
       "7     Original                 0.0                       0.0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SubjectiveScores(df, save_path='./subjective_scores/public_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "155f08d8-8dbb-4fdd-951d-6c69516c3b2e",
   "metadata": {},
   "source": [
    "# PRIVATE TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0f0762dc-40f8-4219-b530-bfb26a447233",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./subjective_votes/final.csv')\n",
    "df = df[df['subset'] == 'private']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5d5dfaa8-21b8-45c7-ba52-f8a5ce4fadc5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Bradley-Terry\n",
      "Questions: 8400\n",
      "Sequences: 30\n",
      "Methods: 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>subjective_score</th>\n",
       "      <th>confidence_95_to_original</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ShannonLab</td>\n",
       "      <td>2.149161072549972</td>\n",
       "      <td>0.1410909324302447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DeepView</td>\n",
       "      <td>1.9977632880170038</td>\n",
       "      <td>0.1389276131620625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nobody</td>\n",
       "      <td>1.4523420234497548</td>\n",
       "      <td>0.13361989469524788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ChouPiJiang</td>\n",
       "      <td>1.3712632482264946</td>\n",
       "      <td>0.13312261928477778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>VQE</td>\n",
       "      <td>1.345525457113669</td>\n",
       "      <td>0.13297958157483583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ByteMM</td>\n",
       "      <td>0.9598655623930443</td>\n",
       "      <td>0.1316983943152462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>TACO_SR</td>\n",
       "      <td>0.6178748949794315</td>\n",
       "      <td>0.13203252195739093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Original</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name    subjective_score confidence_95_to_original\n",
       "0   ShannonLab   2.149161072549972        0.1410909324302447\n",
       "1     DeepView  1.9977632880170038        0.1389276131620625\n",
       "2       Nobody  1.4523420234497548       0.13361989469524788\n",
       "3  ChouPiJiang  1.3712632482264946       0.13312261928477778\n",
       "4          VQE   1.345525457113669       0.13297958157483583\n",
       "5       ByteMM  0.9598655623930443        0.1316983943152462\n",
       "6      TACO_SR  0.6178748949794315       0.13203252195739093\n",
       "7     Original                 0.0                       0.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SubjectiveScores(df, save_path='./subjective_scores/private_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb56b048-062b-441c-a50d-cadd9975a607",
   "metadata": {},
   "source": [
    "# FINAL RESULTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fe17be9a-4b40-41b4-af90-8a4a813d7663",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./subjective_votes/final.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a83784fc-876e-4b9d-be3a-1dccb870e875",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running Bradley-Terry\n",
      "Questions: 42000\n",
      "Sequences: 150\n",
      "Methods: 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>subjective_score</th>\n",
       "      <th>confidence_95_to_original</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ShannonLab</td>\n",
       "      <td>1.848477609480369</td>\n",
       "      <td>0.059886355235425626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DeepView</td>\n",
       "      <td>1.5783185144102103</td>\n",
       "      <td>0.05838462149593195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nobody</td>\n",
       "      <td>1.3054176144490077</td>\n",
       "      <td>0.05733101259184353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ChouPiJiang</td>\n",
       "      <td>1.1401993342524521</td>\n",
       "      <td>0.05690012071439701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>VQE</td>\n",
       "      <td>1.0995012893194431</td>\n",
       "      <td>0.056817301556095344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ByteMM</td>\n",
       "      <td>0.7611459548965849</td>\n",
       "      <td>0.056491140357012505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>TACO_SR</td>\n",
       "      <td>0.6173373192537054</td>\n",
       "      <td>0.05655868626354062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Original</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name    subjective_score confidence_95_to_original\n",
       "0   ShannonLab   1.848477609480369      0.059886355235425626\n",
       "1     DeepView  1.5783185144102103       0.05838462149593195\n",
       "2       Nobody  1.3054176144490077       0.05733101259184353\n",
       "3  ChouPiJiang  1.1401993342524521       0.05690012071439701\n",
       "4          VQE  1.0995012893194431      0.056817301556095344\n",
       "5       ByteMM  0.7611459548965849      0.056491140357012505\n",
       "6      TACO_SR  0.6173373192537054       0.05655868626354062\n",
       "7     Original                 0.0                       0.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SubjectiveScores(df, save_path='./subjective_scores/final_results.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
